{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tempfile\n",
    "import tensorflow as tf\n",
    "\n",
    "from tf_rl.controller import DiscreteDeepQ, HumanController\n",
    "from tf_rl.simulation import KarpathyGame\n",
    "from tf_rl import simulate\n",
    "from tf_rl.models import MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/tmp/tmpdzxofD\n"
     ]
    }
   ],
   "source": [
    "LOG_DIR = tempfile.mkdtemp()\n",
    "print(LOG_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tf_rl.simulation import DiscreteHill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Tensorflow business - it is always good to reset a graph before creating a new controller.\n",
    "tf.ops.reset_default_graph()\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "# This little guy will let us run tensorboard\n",
    "#      tensorboard --logdir [LOG_DIR]\n",
    "journalist = tf.train.SummaryWriter(LOG_DIR)\n",
    "\n",
    "# Brain maps from observation to Q values for different actions.\n",
    "# Here it is a done using a multi layer perceptron with 2 hidden\n",
    "# layers\n",
    "brain = MLP([4,], [10, 4], \n",
    "            [tf.tanh, tf.identity])\n",
    "\n",
    "# The optimizer to use. Here we use RMSProp as recommended\n",
    "# by the publication\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate= 0.001, decay=0.9)\n",
    "\n",
    "# DiscreteDeepQ object\n",
    "current_controller = DiscreteDeepQ((4,), 4, brain, optimizer, session,\n",
    "                                   discount_rate=0.9, exploration_period=100, max_experience=10000, \n",
    "                                   store_every_nth=1, train_every_nth=4, target_network_update_rate=0.1,\n",
    "                                   summary_writer=journalist)\n",
    "\n",
    "session.run(tf.initialize_all_variables())\n",
    "session.run(current_controller.target_network_update)\n",
    "# graph was not available when journalist was created  \n",
    "journalist.add_graph(session.graph_def)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Game 9900: iterations before success 12. Pos: (-3, 7), Target: (-3, 7)\n"
     ]
    }
   ],
   "source": [
    "performances = []\n",
    "\n",
    "try:\n",
    "    for game_idx in range(10000):\n",
    "        game = DiscreteHill()\n",
    "        game_iterations = 0\n",
    "\n",
    "        observation = game.observe()\n",
    "\n",
    "        while game_iterations < 50 and not game.is_over():\n",
    "            action = current_controller.action(observation)\n",
    "            reward = game.collect_reward(action)\n",
    "            game.perform_action(action)\n",
    "            new_observation = game.observe()\n",
    "            current_controller.store(observation, action, reward, new_observation)\n",
    "            current_controller.training_step()\n",
    "            observation = new_observation\n",
    "            game_iterations += 1\n",
    "        performance = float(game_iterations - (game.shortest_path)) / game.shortest_path\n",
    "        performances.append(performance)\n",
    "        if game_idx % 100 == 0:\n",
    "            print \"\\rGame %d: iterations before success %d.\" % (game_idx, game_iterations),\n",
    "            print \"Pos: %s, Target: %s\" % (game.position, game.target),\n",
    "except KeyboardInterrupt:\n",
    "    print \"Interrupted\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fae08076550>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEACAYAAABbMHZzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuUVNWZ9/HvIyCKN0QUubRCBFQUBVEkxkiZMV4YhpmM\nGp1JIJoJuhwTL1HH6wyYiTHrXXFwjEYxGkOMUTJoFBW8UxqNooaLCCKiEAG5CrZyE7r7ef/YVVR1\ndXV3dXdVn7r8PmvVqlPn7D77qd3dzzm1z6m9zd0REZHKslvUAYiISPtT8hcRqUBK/iIiFUjJX0Sk\nAin5i4hUICV/EZEK1DGXQma2HPgcqAV2uvvwjO0x4Ango8SqR939p/kLU0RE8imn5A84EHP3jU2U\nedndx+QhJhERKbCWdPtYG7eLiEiRyDX5O/CCmb1tZuMb2X6Smc03sxlmNih/IYqISL7l2u3zNXdf\nbWYHAs+b2WJ3/3Pa9jlAlbtvNbOzgMeBgfkOVkRE8sNaOraPmU0ANrv7bU2UWQYMS79GYGYaREhE\npBXcPe/d6s12+5hZFzPbJ7G8F3A6sCCjTA8zs8TycMJBpcHFYXfXw50JEyZEHkOxPNQWagu1RdOP\nQsml26cH8KdEbu8IPOTuz5nZxYmEPhk4B7jEzGqArcD5BYpXRETyoNnk7+7LgCFZ1k9OW74LuCu/\noYmISKHoG74RiMViUYdQNNQWKWqLFLVF4bX4gm+rKzLz9qpLRKRcmBkexQVfEREpP0r+IiIVSMlf\nRKQCKfmLiFQgJX8RkQqk5C8iUoGU/EVEKpCSv4hIBVLyFxGpQEr+IiIVKPLkv3p11BGIiFSeSJP/\n8uXQqxesXx9lFCIilSfS5L9uXXhesiTKKEREKk8kyf+aa2DLFqirC6/HZ5sSXkRECianIZ3NbDnw\nOVAL7HT34VnK3AGcRZjJ6wJ3n5ux3d2dTZugWzc48EDYsQOqq8P2rVuhY0fo1Kmtb0lEpHwUakjn\nXKZxBHAg5lnm5QUws1FAf3cfYGYnAncDI7KV3ZjYQ2Y/f5cu4Xn+fDjmmByjEhGRVmlJt09TR54x\nwBQAd58NdDWzHtkK9u/fdCXHHdeCiEREpFVyTf4OvGBmb5tZth763sCKtNcrgT6ZhXbubL6iu+/O\nMSIREWm1XLt9vubuq83sQOB5M1vs7n/OKJP5yaDBxYTdd5+4a/mWW2IMHRrjlVfg//4vnPHvuy9Y\n3nu2RERKRzweJx6PF7yeFs/ha2YTgM3uflvaunuAuLs/kni9GBjp7mvTynjyePD22zBsWMN9X3op\nDBoUnkVEJMI5fM2si5ntk1jeCzgdWJBRbDowLlFmBPBZeuJPuvlmcM+e+AH22AO2bWtR/CIi0grN\nnvmbWT/gT4mXHYGH3P1WM7sYwN0nJ8rdCZwJbAEudPc5Gfvx1audgw9uvK4TT4Q33wwHCBERKdyZ\nf4u7fVpdkZlXVzv77tt4md/+Fi68UMlfRCSpLJL/zp1OxyYuMW/bFu7337ABDjigXcISESlqkfX5\n51NTiR9gzz3D84oVTZcTEZG2iXxI52yGDoWRI1NDP4iISH4VZfIHeOUVeOedqKMQESlPRZf8p0xJ\nLX/5ZXRxiIiUs6JL/uPGpZaV/EVECqPokj9A377heceOSMMQESlbRZn8J0wIzzU10cYhIlKu2vU+\n/5bUZQYDB8L77xcwKBGRIhf1ZC7t7vDDYfDgqKMQESlPRX3mD2GeXw3zLCKVqiy+4dsSP/pReN5r\nrzC/r4iI5E/RJv+rrgrP27bB9OnRxiIiUm6Kttsn/ExqWSN9ikglqrhuH4Brr00tb94cXRwiIuWm\nqJP/rbfCs8+G5SefjDYWEZFyklPyN7MOZjbXzBqkYDOLmVl1YvtcM7spX8GZwemnw+jRuuNHRCSf\ncj3zvxxYRHIG9oZedvehicdP8xNayqBBYXpHERHJj1wmcO8DjALuAxo7/y7oefnxx8OkSbroKyKS\nL7mc+U8CrgHqGtnuwElmNt/MZpjZoLxFl/DNb4ZnTe4iIpIfTQ7vYGajgXXuPtfMYo0UmwNUuftW\nMzsLeBwYmK3gxIkTdy3HYjFiscZ2WV/XrtCvH6xfH5ZFRMpVPB4nHo8XvJ4m7/M3s58BY4EaYA9g\nX+BRdx/XxM8sA4a5+8aM9S2+z7/+z0OvXrBqVat3ISJSciK5z9/db3D3KnfvB5wPvJSZ+M2sh1m4\nF8fMhhMOKBuz7K7NOncuxF5FRCpPS+/zdwAzu9jMLk6sOwdYYGbzgNsJB4m8mzULevcuxJ5FRCpP\nUQ/vkG7JEhg1CpYuzWNQIiJFriKHd0jXqRPs3Bl1FCIi5aFkkr8ZfPxxGOVTRETapmSSf/Ksf/Xq\naOMQESkHJZP8998/PI9r9CZTERHJVckk/+7dw62e6vcXEWm7kkn+AL/7nZK/iEg+lFTyHzZMt3qK\niORDSSX/vn3hiy9g0aKoIxERKW0llfw7dIATToAFC6KORESktJVU8gcYOhQ2bYo6ChGR0lZyyX+f\nfULXj4iItF7JJf+994bNm6OOQkSktJVc8t9nH/j886ijEBEpbSWX/Hv2hE8+iToKEZHSVnLJ/5BD\nYMWKqKMQESltJZf8q6rC6J4iItJ6OSV/M+tgZnPN7MlGtt9hZh+Y2XwzG5rfEOvr1QvWrdMwDyIi\nbZHrmf/lwCIS0zimM7NRQH93HwBcBNydv/Aa6tQJevRQv7+ISFs0m/zNrA8wCrgPyDaV2BhgCoC7\nzwa6mlmPfAaZSV0/IiJtk8uZ/yTgGqCuke29gfRLsCuBPm2Mq0m9eunMX0SkLTo2tdHMRgPr3H2u\nmcWaKprxOutM7RMnTty1HIvFiMWa2mXj9t1XX/QSkfIUj8eJx+MFr8fcs+bpsNHsZ8BYoAbYA9gX\neNTdx6WVuQeIu/sjideLgZHuvjZjX95UXS1x2WXQv394FhEpZ2aGu2frcm+TJrt93P0Gd69y937A\n+cBL6Yk/YTowLhHkCOCzzMSfb927w9qC1iAiUt5aep+/A5jZxWZ2MYC7zwA+MrOlwGTg3/MbYkM9\nesCGDYWuRUSkfDXZ55/O3V8GXk4sT87Y9sM8x9Wk5OBur7wCp5zSnjWLiJSHkvuGL4TB3WbPhpEj\nIU+XEUREKkpJJv8DD4QPPwzLut9fRKTlSjL59+yZWl6zJro4RERKVcknf33ZS0Sk5Uoy+XfuDA8+\nGJa3bIk2FhGRUlSSyR9g9OjwvHhxtHGIiJSikk3+XbvC1VdraGcRkdYo2eQPcNhhsHBh1FGIiJSe\nkk7+778PTz8ddRQiIqWnpJP/GWeE59raaOMQESk1TY7qmdeK8jiqZ/39wrJl0Ldv3nctIhK5SEb1\nLAVdu0J1ddRRiIiUlpJP/oMGwRdfRB2FiEhpKfnkv2wZvP561FGIiJSWkk/+I0bAp59GHYWISGlp\nNvmb2R5mNtvM5pnZIjO7NUuZmJlVm9ncxOOmwoTb0FFHQZcu7VWbiEh5aHYyF3ffbmanuvtWM+sI\nvGpmJ7v7qxlFX3b3MYUJs3GdO8O2be1dq4hIacup28fdtyYWdwc6ABuzFMv7rUi56NwZduyIomYR\nkdKVU/I3s93MbB6wFpjl7osyijhwkpnNN7MZZjYo34E2pnNn+PLL9qpNRKQ85HrmX+fuQ4A+wClm\nFssoMgeocvdjgV8Cj+c1yiYo+YuItFzOE7gDuHu1mT0NHA/E09Z/kbY808x+ZWbd3L1e99DEiRN3\nLcdiMWKxWOuiTqPkLyLlJB6PE4/HC15Ps8M7mFl3oMbdPzOzPYFngZvd/cW0Mj2Ade7uZjYc+KO7\n983YT0GGd5g2Df7wB3jssbzvWkQkcoUa3iGXM/+ewBQz243QTfSgu79oZhcDuPtk4BzgEjOrAbYC\n5+c70Mb06QMrV7ZXbSIi5aHkB3ZbuRKGD9dcviJSngp15l/yyb+mJvT7a1hnESlHGtWzER07Ql0d\nbN8edSQiIqWj5JN/ki74iojkriyS//e+p9s9RURaouT7/AGOPhqWLNEwDyJSftTn34QTToCdO6OO\nQkSkdJRF8r/0UhgyJOooRERKR1l0+6xbBz16hLt+LJKxRUVECkPdPk046KDwrBm9RERyUxbJH+CY\nY+Djj6OOQkSkNJRN8j/0UPjb36KOQkSkNJRN8j/kECV/EZFclU3yP/RQdfuIiOSqrJK/zvxFRHJT\nNslf3T4iIrkri/v8IXWvf00NdOhQsGpERNpVJPf5m9keZjbbzOaZ2SIzu7WRcneY2QdmNt/MhuY7\nyFwcdBAccACsWhVF7SIipaXJ5O/u24FT3X0IcAxwqpmdnF7GzEYB/d19AHARcHehgm3OkUfCsmVR\n1S4iUjqa7fN3962Jxd2BDsDGjCJjgCmJsrOBrokJ3dvdV74CH34YRc0iIqWl2eRvZruZ2TxgLTDL\n3RdlFOkNrEh7vRLok78QczdwILz/fhQ1i4iUllzO/OsS3T59gFPMLJalWObFiPa5ipzhkEN0r7+I\nSC465lrQ3avN7GngeCCetmkVUJX2uk9iXQMTJ07ctRyLxYjFYrlHmoMBA8KkLiIipSoejxOPxwte\nT5O3eppZd6DG3T8zsz2BZ4Gb3f3FtDKjgB+6+ygzGwHc7u4jsuyroLd6AmzcCH37QnW1hnYWkfJQ\nqFs9mzvz7wlMMbPdCF1ED7r7i2Z2MYC7T3b3GWY2ysyWAluAC/MdZK66dYMvvoBFi+Coo6KKQkSk\n+JXNl7xS9cC998L48QWvSkSk4Ap15l92yX/UKJg5E9rpbYmIFJSSf44WLw5f9qqthd3KZuQiEalU\nmsYxR0ccEYZ52LAh6khERIpX2SV/gJ49YfXqqKMQESleZZn8e/VS8hcRaUrZ9fmHusKzLvqKSKlT\nn38LHH101BGIiBS3skz+DzwAQyOZVUBEpDSUZfI/9FBYvjzqKEREildZJv/u3cN0jps2RR2JiEhx\nKsvkbwaHHQZLl0YdiYhIcSrL5A/QuTMMHx51FCIixalsk/9110FVVfPlREQqUdkm/6qq0PcvIiIN\nlW3y33132LEj6ihERIpT2Sb/zp3hyy+jjkJEpDg1m/zNrMrMZpnZQjN718wuy1ImZmbVZjY38bip\nMOHmTmf+IiKNy2UC953Ale4+z8z2Bv5qZs+7+3sZ5V529zH5D7F1unSBLVuijkJEpDg1e+bv7mvc\nfV5ieTPwHtArS9GimjL9wAPDmf+6dVFHIiJSfFrU529mfYGhwOyMTQ6cZGbzzWyGmQ3KT3itZwbH\nHgvz50cdiYhI8cml2weARJfPNODyxCeAdHOAKnffamZnAY8DAzP3MXHixF3LsViMWCzWipBz16MH\nVFcXtAoRkbyKx+PE4/GC15PTeP5m1gl4Cpjp7rfnUH4ZMMzdN6ata7fx/JN+8AM48UQYP75dqxUR\nyZvIxvM3MwPuBxY1lvjNrEeiHGY2nHBQ2ZitbHvq3RtWrow6ChGR4pNLt8/XgO8C75jZ3MS6G4BD\nANx9MnAOcImZ1QBbgfMLEGuL9esHL70UdRQiIsWn2eTv7q/SzCcEd78LuCtfQeVLv36wbFnUUYiI\nFJ+ynMM3aeXKMMbPmjXh4q+ISKnRHL6t0KdPeH7uuWjjEBEpNmWd/AHOPhs6dYo6ChGR4lL2yb+q\nClatijoKEZHiUvbJf//9NZeviEgmJX8RkQpU9sm/e3f49NOooxARKS5ln/z33RemTo06ChGR4lL2\nyb9///C8fXu0cYiIFJOyT/6HHw49e6rrR0QkXdknfwj9/prURUQkpSKSf00NTJkSdRQiIsWjrMf2\nSdUdniOqXkSk1TS2TxsUeMIwEZGSUxHJf9q08HzPPdHGISJSLCoi+R9wAFx6KSxfHnUkIiLFIZdp\nHKvMbJaZLTSzd83sskbK3WFmH5jZfDMbmv9Q22b0aHjhhaijEBEpDrmc+e8ErnT3o4ARwKVmdmR6\nATMbBfR39wHARcDdeY+0jQ47DP76V9iyJepIRESi12zyd/c17j4vsbwZeA/olVFsDDAlUWY20NXM\nimrurAEDwvPChdHGISJSDFrU529mfYGhwOyMTb2BFWmvVwJ92hJYIYwcCc88E3UUIiLRa3YC9yQz\n2xuYBlye+ATQoEjG6wZ31U+cOHHXciwWI9bO92COHw/Tp7drlSIiLRKPx4nH4wWvJ6cveZlZJ+Ap\nYKa7355l+z1A3N0fSbxeDIx097VpZSL7klfSs8/CbbdpTl8RKR2RfcnLzAy4H1iULfEnTAfGJcqP\nAD5LT/zFYq+94LXXoo5CRCR6zZ75m9nJwCvAO6S6cm4ADgFw98mJcncCZwJbgAvdfU7GfiI/86+u\nhq5d4bPPYL/9Ig1FRBqxfTvssUfUURSPQp35V8TYPvXjgEmT4Ioroo5EpDJ88UWYVGnrVthzz+xl\n3GG3tH6IurrUmFyVTmP75NGVV4Y/rN/+NupIRMrXlCmwc2dI/ADf/nbjZWfOrP/6qqsKF5cEFZf8\n0yd1ufDC6OIQKVfjx4eTqwsugEcegQ4d4M474amnGh9Z95NP4OSTU9snTWq3cCtWxSX/bt1C3//r\nr8PAgVFHI1Je3n4b7rsv9XrcODj66NSJ1qZN0K9f/TJjx4YDxquvhtfJLtnq6vaJuVJVXPKH8DF0\nwABYsgQ2bIg6GpHyMXZseB4yJNWHP38+dOkChx4KK1aEARZffDFse+IJ+P3vw3Jy9N3k14E0Cm9h\nVdwF36TkBaZZszTev0hbrF8P550X/pcg3FK9eTNs2xaS/rPPwumnN7yAe+ONMGMGzJ0b9tG9e2pb\nsuwDD4RPDZs3h/0Wo/feC/OEd+1amP3rbp8CGDcOTj1Vff8irbFxIxxySMPBEt98E044oWH5xu7e\nuffe0O2TbuvWhsm+yNLHLmYhlxRqqljd7VMA/fvD0qVRRyFSOmprQ7IzC/NkJBP/P/4jzJkTzuCz\nJX6ABQtCf797mFc7Kdv/YJcuDZP/li3hLqBzz4V2GP2gWb/8ZeqAVlcXbSytUdFn/r//PTz9NDz8\ncNSRSCVbvx4OPDDqKHIzdSqcf379dRs2hANBS/3kJzBhQsMun6SNG8Nt2ZMmZd9/FOnk3XfhoovC\nbeKHH15/W21t/e8q5IvO/Atgr73CtwmltHzwQWH3v2ULHHxww/V//Svccgs8+ij8wz+E2xfTtebs\nb8YMOOigcAHULNSRrczNN7d83/nkHrpzpk0Lt24eeWS4NbO2tnWJH+C//ivsN1vih3Bn3pQp4fmf\n/imsSz/wvPlm6+ptiW98I/VJxwwGDw53CiYT/wMPwGOPheWVK+HDD0OblAR3b5dHqKq4PP64+5gx\nUUchLbH77u4hZbg//LD7GWe4X3GF+8SJ9ctNmeL+xRetq2PRorD/hx4Kr5P1DRyYWk4+3nknlJk+\nPbxetKhldWXuD9x37Ehtr61Nra+tbd37yWbbNvcHH2y6zJYt7qtXh+U77kjFscceIZZ8xpOL2bPD\n86ZN9dtr+/bC1Hfuudl/P7vtllpOyizz85+7b9zoXl3d9jgSuTP/ObkQO81aUREm/1tvrf8LlOLz\nt7+533KL+5NPuq9fX/8fbMCAhv90H3zgftttqdd1dS2vMz3RLVyYPQEsX55a/s536m+rq3P/7LPc\n6gL3vn3db7jBfezY1D4uv9x9/PiG7y3dnXe6v/9+83Xs3JlqhzfeaPheLrigfvlly9xXrEhtnzcv\n/A6Sr4cMye29FdJPftKwzdsivY3eeiu0f/q+H3ggJPTkwWbHjvqJ/YYbsv+dVFW1LS53Jf+C+MEP\nQgts2RJ1JKVl0SL3PffMzx92UzZsyP4PdeWV2ddne3z1q03XsXVrarmuzv3zz927dnUfOrT+fl54\nwRuclY8c2XTdzXn1VfcjjnB/6qnUumSCaWqf6Z8GTj65+XqSZbt0aXy/3/mO+69+Vf+glvn4xS+a\nr6u91NS4v/uu+4cfhtjacoa9alXj73nTptz2kTxYrljh/tprqZ+/667Wx5Wk5F8AyY/qP/pR1JEU\nr2wfqfv3T/1xX3FF4eq+//7s/5AffOA+a5bvOgtzD4l74sRUmQUL3P/u78Ly559n3/977/muM1v3\n+nWce244e4fwz9yYjz5K/czOnfX3kewy2bYtJJGnngpxJyXLvfRS/X1efHHDhH/ZZWE5W5s0Zd68\nhuWrq90feyzE9OWXTR9s0n/+mmuarisqnTuH32VrZTuIH364+403tm5/NTWp/axc2fq4kpT8C+Sm\nm9x//OOooyhOffqk/oivuy6se/LJ8PrNN1PbTjklbLvppuaTUa527nQfPNj9qqta/rPJj++1te49\ne7q/+GLoIknGm7wW8Jvf1E+g6f/8X36Ze321teEf3j0kkm99y71jR/c//SmsGzy4/r6ffTZV3+jR\nDfvO6+rCCUlyn0np+3jwQfc5c+ofeJJeein8TpLdmslHv37Zu0fWrg3JLjMBnnBC2J781NPUQTBK\nENq7tQ491H3SpLC8ZYv7PffkJay8UfIvkJtvDq1wxhlRR1J46QmiOelntNkedXX1E2rmY+HCtnWn\nHXxw2M+117Z+H+7uZ5/dMPkmDwCjR6deP/powyTaFsn9/uUv2dunX7/wnN7t1JwtW9w7dQpJ3919\n6dLU/tLbOrOueDy3/a9Z437eeeHaykMP1f+UUsx+9rPwPj/6qOU/m+xCe+SR/MeVL0r+BbJmTeqf\npFev8JzLRbRSkt6Pm4u6uvrlb7yxfjLZti2sT2+7xhJc0mefhcTS2B0is2aFRL9xo3vv3ql9PPFE\nq9+2u7sfd1xqX9Omhf1BqmslfXs+/0TPO6/+fh97LKz//PP81VdX53766WE/n3xS/32B+5lnhusm\n5S69m6W5O5gyffhh+IRbzCJL/sBvgLXAgka2x4BqYG7icVMj5QrXOm20fn34iFuIJFAMxoxJva/0\nM9u333Y/9VT3556rf3fK//xPKHvLLeH1xx+nfv7UU7PXsWZNavnVV1PlZ8xw/+d/Tr2++273005z\nnzq1/s8PG9bw4JE8yLTFmjXu++wT7nJJSq+jtjZ1cLzhhrbXl/TMM2Gf3buHC4qZZs1ynzkzP3WB\n+/nn139f+foEUyomT67//u+8M6z/3e/qX5eqqwsJ/7XX3H/5y1B20KBoYs5VlMn/68DQZpL/9Bz2\nU6CmyY/Nm92///1UN9D69WH9zJnuZu7/+Z8hSZQicD/66PoHtqbucMh2AGzpQXHHjub3X1eX/c6d\n//7vcCZbKBdcEOpJv56QfhdPqUm2W/onpkqU+Xd00kmp5bVrc/9bLzaRdvsAfZtJ/k/msI/CtEye\npXd5ZD7OOSfq6JpXV9ewrx3CGXAy0cZiqffUrVvDLgpo+AWpefNa3qf61lup/a1Y4X7MMU3/A7bX\n2er27eHTQLl0iSS/H3DGGfn7NFGKXnnF/ac/DXczNXfikXy8+mq4ZbSYFXPyHwl8CswHZgCDGilX\nsMbJt+RtdcnH1VeHs4jdd3d//fWoo2tc+v3fEBLctdf6ru6N5O2RyUd6N8f27e4nnpj6xJMvdXWp\nfv733gv97rW14UszELqE5s0r7nYtdjt2hGsK5XIwy4elS92XLKn/CXS//dwXL07dgrtgQbQx5qpQ\nyT+ngd3MrG/i7H5wlm37ALXuvtXMzgL+190bzJFlZj5hwoRdr2OxGLEiHUh/2zZYty6M8dKvX5h0\n+umnYfToVJk774RLLinMQE65+PWvQ93/9m+pdSNHwiuvZC+f/DW/9FIYlGrKFE2QLZXjj3+Eo44K\nj2IXj8eJpw1bevPNN+NRjeffVPLPUnYZMMzdN2as91zqKmbLl8O//Au88UZq3de/Hiar2HPPMLBX\nXR107FjYOGpqoFOnsJwc9bBXrzAP6n33wYgRYXKJqVNhv/3CYFhRHaREpG0incylmTP/HsA6d3cz\nGw780d37ZilX8sk/qbo6jESYHJP8lFNCEv7Wt8LrqVPh298uXP0PPQTf/W4YN/2tt+pvK5MmFpGE\nyJK/mT1M6NfvTrjlcwLQCcDdJ5vZpcAlQA2wFfixu7+RZT9lk/yT0s/A033lK2Fo13wzg2OPDXOi\nptu+PcyP+vLLDccYF5HSpmkci9SVV8Ltt4flSy8Nk08fdlgYo33p0jA/6ZAhDX9u06Zw4Nh779zq\nca/fdfOLX8DVV8PMmXDmmW1+GyJSpJT8i5x76gJq5oXUTz8NE1IkrVsHPXqkfu6hh2D//WHUqMb3\nn9zn88+HCecLfV1BRIqDZvIqcukJ//77w10Fye6ZzJmOqqtTyz17hv77v//7+p8Qrr8+7HPlytS6\nbt3gtNOU+EWk7ZT8C+D73w9zfQ4eDNdcE9YtXBiS+Z//DDt3whFHhPVr1oTnsWPDweLhh8P2n/88\nrK+qChNFQ3FMWi0i5UHdPu2gsfvpd+4M9x+ffTZ07tyw3Ne/Hg4WSXV1ujdfpNKo26eEJe8IOuec\n1Lrjjw/dN//6ryHxA8yZk9p+5ZXhC1vJ20nvvVeJX0TyR2f+7aCmJnxbeJ99cvuy1WuvwUknKdmL\niO72ERGpSOr2ERGRvFHyFxGpQEr+IiIVSMlfRKQCKfmLiFQgJX8RkQqk5C8iUoGU/EVEKlCzyd/M\nfmNma81sQRNl7jCzD8xsvpkNzW+IIiKSb7mc+T8ANDpdiJmNAvq7+wDgIuDuPMVWtuIannMXtUWK\n2iJFbVF4zSZ/d/8zsKmJImOAKYmys4GuiXl9pRH6w05RW6SoLVLUFoWXjz7/3sCKtNcrgT552K+I\niBRIvi74Zg46pBHcRESKWE6jeppZX+BJdx+cZds9QNzdH0m8XgyMdPe1GeV0QBARaYVCjOqZj9lg\npwM/BB4xsxHAZ5mJHwoTvIiItE6zyd/MHgZGAt3NbAUwAegE4O6T3X2GmY0ys6XAFuDCQgYsIiJt\n126TuYiISPFol2/4mtmZZrY48UWwa9ujzvZkZlVmNsvMFprZu2Z2WWJ9NzN73syWmNlzZtY17Weu\nT7THYjM7PW39MDNbkNj2v1G8n3wwsw5mNtfMnky8rsi2MLOuZjbNzN4zs0VmdmIFt8X1if+RBWb2\nBzPrXClPzWGEAAADYUlEQVRtke3Lsvl874m2nJpY/4aZHdpsUO5e0AfQAVgK9CV0F80Djix0ve35\nAA4GhiSW9wbeB44E/h/wH4n11wI/TywPSrRDp0S7LCX1KexNYHhieQZwZtTvr5Vt8mPgIWB64nVF\ntgXhOzDfTyx3BParxLZIvJ+PgM6J11OB71VKWwBfB4YCC9LW5e29A/8O/CqxfB7wSLMxtcOb/irw\nTNrr64Drov5lFPg9Pw6cBiwGeiTWHQwsTixfD1ybVv4ZYATQE3gvbf35wD1Rv59WvP8+wAvAqYS7\nxKjEtkgk+o+yrK/EtuhGOCnan3AQfBL4ZiW1RSKRpyf/vL33RJkTE8sdgfXNxdMe3T7ZvgTWux3q\njUTittihwGzCLzZ559NaIPnN516EdkhKtknm+lWUZltNAq4B6tLWVWJb9APWm9kDZjbHzH5tZntR\ngW3h7huB24CPgU8IdwU+TwW2RZp8vvddedbda4BqM+vWVOXtkfwr5oqyme0NPApc7u5fpG/zcEgu\n+7Yws9HAOnefS8Mv/wGV0xaEM7DjCB/HjyPcDXddeoFKaQszOwy4gnD22wvY28y+m16mUtoimyje\ne3sk/1VAVdrrKuofvcqCmXUiJP4H3f3xxOq1ZnZwYntPYF1ifWab9CG0ySrqD43RJ7GulJwEjDGz\nZcDDwDfM7EEqsy1WAivd/a3E62mEg8GaCmyL44G/uPuniTPTxwhdwpXYFkn5+J9YmfYzhyT21RHY\nL/Fpq1HtkfzfBgaYWV8z251wMWJ6O9TbbszMgPuBRe5+e9qm6YSLWiSeH09bf76Z7W5m/YABwJvu\nvgb4PHFHiAFj036mJLj7De5e5e79CH2SL7n7WCqzLdYAK8xsYGLVacBCQn93RbUFoX97hJntmXgP\npwGLqMy2SMrH/8QTWfZ1DvBis7W304WOswgXe5YC10d94aUA7+9kQv/2PGBu4nEm4SLXC8AS4Dmg\na9rP3JBoj8XAGWnrhwELEtvuiPq9tbFdRpK626ci2wI4FngLmE84292vgtviPwgHvwWEu6A6VUpb\nED4FfwLsIPTNX5jP9w50Bv4IfAC8AfRtLiZ9yUtEpAJpGkcRkQqk5C8iUoGU/EVEKpCSv4hIBVLy\nFxGpQEr+IiIVSMlfRKQCKfmLiFSg/w+W9pFU+SNQPAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fae211adbd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "N = 500\n",
    "smooth_performances = [float(sum(performances[i:i+N])) / N for i in range(0, len(performances) - N)]\n",
    "\n",
    "plt.plot(range(len(smooth_performances)), smooth_performances)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.6869445151941282"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.average(performances[-1000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.colorbar.Colorbar instance at 0x7fa050305dd0>"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAJBCAYAAABRfpDFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGgJJREFUeJzt3X+wZGV95/HPhzsgDojEUAGBSU3WRYNGUYpfhUm8GLAm\nxJ8pE0MkGkxZVrJjTKqSqCQxQ2o3WZO4IVk2LiKiBCOpoBgMjmRivCpRR0aZAWeGBTRUZoZlQFkw\ngETuzHf/6AYu95zb/fTTp/ucnuf9quqy+/bpc773yvnO5zzn6XMcEQKAYQ5quwAAs4FmASAJzQJA\nEpoFgCQ0CwBJaBYAktAsgBlk+1Dbm21vtb3D9h/XLPNG29ts32L7X2y/aJxtrhrnwwDaERGP2j4r\nIh6xvUrSjbZ/PCJuXLLYtyT9ZEQ8aHudpA9IOiN3mzQLYEZFxCP9p4dImpN0/7L3v7zk5WZJx4+z\nPQ5DgBll+yDbWyXtlfS5iNgxYPFfkfTpcbZHswBmVETsj4gXq5cYftL2fN1yts+S9BZJ7xxnexyG\nABlsT/1LVRHhFX7+oO3rJZ0iaWHpe/1BzcskrYuI/zfO9mkWQKYNLW7L9lGSFiPiAdtPl3SOpIuW\nLfPDkj4h6fyIuHPcGmgWwGx6tqSP2D5IveGEv46Iz9p+myRFxKWS3iPpByS937YkPRYRp+Vu0HxF\nHRid7fivU9ze72nlw5BpYYATQBIOQ4BMB7ddwJSRLAAkIVkAmUrbeUgWAJKU1hyBxjBmAQA1SBZA\nptJ2HpIFgCSlNUegMYxZAEANmgWAJByGAJlK23lIFgCSlNYcgcYwwAkANUgWQKbSdh6SBYAkpTVH\noDGMWQBADZIFkIlkAQA1SBZAptJ2HpIFgCQ0CwBJSktSQGMY4ASAGiQLIFNpOw/JAkCS0poj0BjG\nLACgBskCyFTazkOyAJCktOYINIYxCwCoQbMAkITDECBTaTsPyQJAkk42C9vrbN9m+w7b72y7nmFs\nf8j2Xtu3tl1LKttrbH/O9nbb37D9623XNIjtQ21vtr3V9g7bf9x2TQdP8dEFnWsWtuckXSJpnaTn\nSzrP9ontVjXUFerVO0sek/SbEfECSWdI+i9d/jtHxKOSzoqIF0t6kaSzbP94y2UVpYuHXadJujMi\n7pIk21dLeo2knW0WNUhEfNH22rbrGEVE3CPpnv7zh2zvlHSsuv13fqT/9BBJc5Lub7GcTu48k9S5\nZCHpOEm7lrze3f8ZJqTf6F4iaXO7lQxm+yDbWyXtlfS5iNjRdk0l6WJzjLYLKIntwyVdI+kdEfFQ\n2/UMEhH7Jb3Y9jMl3WB7PiIW2qqnK2MJ09LFZLFH0polr9eoly7QMNsHS/q4pKsi4pNt15MqIh6U\ndL2kU9qupSRdTBZbJJ3Qj8Z3S3qDpPPaLOhAZNuSLpe0IyIubrueYWwfJWkxIh6w/XRJ50i6qM2a\nSBYti4hFSesl3SBph6S/jYjODrpJku2PSfqSpOfa3mX7grZrSvBSSeerd1bh5v6jy2d0ni3pn/tj\nFpslfSoiPttyTUVxBEMEwKhsxz1T3N4xkiLCU9xkReeSBYBuolkASNLFAU5gJhw8zb1ncYrbWgHJ\nAkASkgWQaVVhyWLsX9c2p1NwwGj7jEOXNdQb/6CZ1VQsSJqfwHrXTmCdkvRJSa+d0LpPntB63y/p\nVye07r0TWu+Vkt40gfW+YqSlD56bQAkdxpgFgCSMWQCZpjpm0QEdTxZr2y5gRD/adgEZZvG7WCe1\nXUCROt4b17ZdwIhmsVmc2nYBGbrRLKY6z6IDOp4sAHQFzQJAksKCFNAgTp0CQBXJAshV2N5DsgCQ\npLDeCDSosL2HZAHMoFFuP2n7VNuLtn92nG0W1huBBrW79zx++8mt/Xu/fM32puUXt+7fDvS9kj4j\naaxv1JIsgBkUEfdExNb+84fUu+3ksTWLvl29m0jdN+42SRZAro7Ms1jp9pO2j1PvPsEvV29e/1jX\nnhnaLPr3krhYvT/NByPiveNsEMBwC49IC98bvtyQ209eLOldERH9m0qNdRgy8L4h/eOd/yPpbPVu\nK3iTpPOWHhf1rpQ1qYvfTMratgvIMKmL30zSpC5+MymvSL5Slu2IEyddz5Lt7axexat/+8l/kLSx\n7q5ytr+lJxvEUZIekfTWiLgup4ZhyeI0SXdGxF39jV+tXqzp9B3CgANdyu0nI+I/LVn+CvXu4pbV\nKKThzeI4SbuWvN4t6fTcjQFozOO3n7zF9s39n10o6YclKSIubXqDw5oFF+MFVtLi6YGIuFEjnM2M\niLHvvzvs190jac2S12vUSxfLLCx5vlazOSaA8mzrP5BiWLPYIumE/qmZuyW9QdJ51cXmm60KmIqT\n9NSrbl012sc7cup0WgY2i4hYtL1e0g3q/WkuXz5DDEAZhh51RcRGSRunUAswWwqb0sh0bwBJCuuN\nQIMK23tIFgCSFNYbgQYVdjaEZAEgCc0CQBIOQ4Bche09JAsASQrrjUCDCtt7SBYAkhTWG4EGFbb3\nkCwAJCmsNwINYlIWAFSRLIBche09JAsASQrrjUCDCtt7SBYAkjTSG4+PNzaxmqnZ/boT2i5hZD99\n7SfaLmFkP6jvtF3CSK4a6+Z+B77CghTQIE6dAkAVyQLIVdjeQ7IAkKSw3gg0qLC9h2QBIElhvRFo\nEGdDAKCKZAHkKmzvIVkASFJYbwQaVNjeQ7IAkIRmASBJYUEKaFBhew/JAkCSwnoj0CAmZQFAFckC\nyFXY3kOyAJCksN4INKiwvYdkASBJYb0RaBBnQwCgimYBIAmHIUCuwvYekgWAJIX1RqBBhe09JAsA\nSQrrjUCDCtt7SBYAkhTWG4EGMSkLAKpIFkCuwvYekgWAJIX1RqBBhe09JAsASWgWAJIUFqSABnHq\nFACqSBZArsL2HpIFgCQ0CyDXqik+lrH9Idt7bd+6Unm2523fbPsbthfG/XVpFsBsukLSupXetH2k\npP8l6VUR8WOSXj/uBhs56nqP/rCJ1UzN3LX72i5hZG954cfaLmFkD2+ZrX+Lrhr1Ay2eDYmIL9pe\nO2CRX5T08YjY3V/+2+Nuc7b+3wSQ6gRJz7L9OdtbbP/SuCssbDwXaFC3956DJZ0s6ackrZb0Zdtf\niYg7clfY7V8XKNTCzt5jDLskfTsivifpe7a/IOkkSTQL4EAyf2Lv8biLrh15FX8v6RLbc5KeJul0\nSf9jnJpoFkCuFvce2x+T9DJJR9neJekP1Dv0UERcGhG32f6MpFsk7Zd0WUTsGGebNAtgBkXEeQnL\n/JmkP2tqmzQLIFdhew+nTgEkKaw3Ag3iK+oAUEWyAHIVtveQLAAkKaw3Ag0qbO8hWQBIUlhvBBrE\n2RAAqKJZAEjCYQiQq7C9Z2iysL2mf7Wd7f0Lf/76NAoD0C0pvfExSb8ZEVttHy7pa7Y3RcR4l+YA\nZh3J4qki4p6I2Np//pCknZKOnXRhALplpN7Yv5rwSyRtnkQxwEwp7NRpcrPoH4JcI+kd/YTxhOs2\nbHvi+fPmj9bz5o9prEBgUr7w+dAXvxBtlzEzHDH8j2X7YEn/IGljRFy87L34QJw/ofImY07cN2Qa\nZu2+IYcful8R4ZRlbUd8ZdIVLdneGUqubVJSzoZY0uWSdixvFADKkXIY8lJJ50u6xfbN/Z+9OyI+\nM7mygBlQ2NmQob9uRNwoZnoCxaMJAEhSWJACGlTY3kOyAJCksN4INKiwSVkkCwBJSBZArsL2HpIF\ngCSF9UagQYXtPSQLAEkK641AgzgbAgBVJAsgV2F7D8kCQBKaBYAkhQUpoEGF7T0kCwBJCuuNQIMK\n23sa+XX3+KomVjM1G17edgUZrmu7gNEdtnV/2yWgQYX1RqA5waQsAKgiWQCZ9hW295AsACQprDcC\nzSFZAECNwnoj0JzFuWn+W9v+aWiSBYAkNAsASTgMATLtWzXN3ef7U9xWPZIFgCQkCyDTvrmy5nuT\nLAAkIVkAmfYVdnlvkgWAJCQLINMiyQIAqkgWQKZ9he0+JAsASWgWAJKUlaOABnHqFABqkCyATCQL\nAKhBsgAykSwAoAbNAsi0qLmpPerYXmf7Ntt32H5nzftH2f6M7a22v2H7l8f5fWkWwAyyPSfpEknr\nJD1f0nm2T1y22HpJN0fEiyXNS3qf7eyhB8YsgEwtT/c+TdKdEXGXJNm+WtJrJO1cssz/lfSi/vMj\nJH0nIhZzN0izAGbTcZJ2LXm9W9Lpy5a5TNI/275b0jMk/fw4G6RZAJkmeTbkpoVHtGXhkUGLRMJq\nLpS0NSLmbT9H0ibbJ0XEv+fURLMAOujU+dU6dX71E6//90X3L19kj6Q1S16vUS9dLHWmpP8mSRHx\nTdv/Kul5krbk1MQAJzCbtkg6wfZa24dIeoOk65Ytc5uksyXJ9tHqNYpv5W6QZAFkanNSVkQs2l4v\n6QZJc5Iuj4idtt/Wf/9SSX8k6Qrb29QLBr8TEZWIkopmAcyoiNgoaeOyn1265Pm3Jb2qqe3RLIBM\nXIMTAGqQLIBMXIMTAGqU1RqBBvEVdQCoQbIAMpEsAKAGzQJAkkYOQz4cO4cv1CGffco3e2fDjZef\n03YJozu87QJG5ZGW5jAEAGowwAlkYro3ANQgWQCZmO4NADXKao1AgzgbAgA1SBZAJpIFANQgWQCZ\nmGcBADVoFgCScBgCZGJSFgDUKKs1Ag3i1CkA1CBZAJlIFjVsz9m+2fanJl0QgG5KTRbvkLRD0jMm\nWAswU5iUtYzt4yWdK+mDGvUihQAOGCnJ4s8l/bakIyZcCzBTSptnMfC3tf1KSfdGxM2251da7oEN\nlzzx/ND503To/GmNFQhMzPYFacdC21XMjGGt8UxJr7Z9rqRDJR1h+8qIeNPShY7csH5S9QGT84L5\n3uNxH7+orUpmwsBmEREXSrpQkmy/TNJvLW8UQKk4dTpYTKQKAJ2XPEITEZ+X9PkJ1gLMFJIFANQo\n69wP0CCSBQDUIFkAmZjuDQA1SBZAptKme5MsACQpqzUCDeJsCADUoFkASMJhCJCJwxAAqEGyADIx\nKQsAapAsgExMygKAGmW1RqBBnA0BgBqNJIu7PnxiE6uZmofPm70eedgr9rddwuiuaLuA0Yx6By2S\nBQDUYMwCyESyAIAaNAsASTgMATIx3RsAapAsgExM9waAGmW1RqBBnDoFMBNsr7N9m+07bL9zhWX+\nsv/+NtsvGWd7JAsgU5vJwvacpEsknS1pj6SbbF8XETuXLHOupP8cESfYPl3S+yWdkbtNkgUwm06T\ndGdE3BURj0m6WtJrli3zakkfkaSI2CzpSNtH526QZAFkanmexXGSdi15vVvS6QnLHC9pb84GSRbA\nbIrE5ZZ/mTb1cxUkC6CD7l64Q3cv3DlokT2S1ix5vUa95DBomeP7P8tCswAyTXJS1tHzJ+ro+Sev\nE/P1i25YvsgWSSfYXivpbklvkHTesmWuk7Re0tW2z5D0QERkHYJINAtgJkXEou31km6QNCfp8ojY\naftt/fcvjYhP2z7X9p2SHpZ0wTjbpFkAmdqelBURGyVtXPazS5e9Xt/U9hjgBJCEZAFkajtZTBvJ\nAkASkgWQiWQBADVIFkAmLqsHADVIFkAmLqsHADVoFgCSlJWjgAZx6hQAapAsgEwkCwCoQbIAMjEp\nCwBqkCyATEzKAoAaZbVGoEGcDQGAGjQLAEk4DAEycRgCADVIFkAmJmUBQI1mksUZjaxlav79ac9o\nu4SRHfbog22XMLq/a7uAyWJSFgDUKKs1Ag3ibAgA1CBZAJlIFgBQg2QBZCJZAEANmgWAJByGAJmY\n7g0ANUgWQCamewNAjbJaI9AgTp0CQA2SBZCJZAEANUgWQCbmWSxj+0jb19jeaXuH7Rm7LhaAJqQk\ni7+Q9OmIeL3tVZIOm3BNADpoYLOw/UxJPxERb5akiFiUNIMXgwSax6Ssp/oRSffZvsL2121fZnv1\nNAoD0C3DmsUqSSdL+quIOFnSw5LeNfGqgBmwT3NTe3TBsBy1W9LuiLip//oa1TSLDZc8+Xz+tN4D\n6LqFh6SFh9uuYnYMbBYRcY/tXbafGxG3Szpb0vbly21YP6nygMmZP7z3eNxF9432+a78iz8tKSM0\nb5f0UduHSPqmpAsmWxKALhraLCJim6RTp1ALMFP27S8rWTDdG0CSsk4UAw1aXCRZAEAFyQLItG+x\nrN2HZAEgCc0CQJKychTQoH0McAJAFckCyESyAIAaJAsg0+JjJAsAqCBZAJn27ytr9yFZAAcY28+y\nvcn27bb/0faRKyz3btvbbd9q+29sP23QemkWQK7Fuek9RvMuSZsi4rmSPquaq9vZXivprZJOjogX\nSpqT9AuDVkqzAA48r5b0kf7zj0h6bc0y35X0mKTV/Vt8rJa0Z9BKyzroAprU3XkWR0fE3v7zvZKO\nXr5ARNxv+32S/k3S9yTdEBH/NGilNAtgBtneJOmYmrd+d+mLiAjbUfP550j6DUlr1bsX0N/ZfmNE\nfHSlbTbSLOKlTaxlenz/7N0n6Y62C8gwMNNisK8sSJsXVnw7Is5Z6T3be20f07/g9rMl3Vuz2CmS\nvhQR3+l/5hOSzpQ02WYBFGnRk1v3KWf1Ho/7yz8c5dPXSXqzpPf2//eTNcvcJun3bT9d0qPqXbn/\nq4NWygAncOD575LOsX27pJf3X8v2sbavl564EPeVkrZIuqX/uQ8MWqkjKoczI7Ed+5811iqm7t77\n265gdN9tu4AMs3YYcpakiEiKC7ZD28fbd0byAifXNikkCwBJGLMAci22XcB0kSwAJCFZALlIFgBQ\nRbIAcj3WdgHTRbIAkIRmASAJhyFArn1tFzBdJAsASUgWQC5OnQJAFckCyEWyAIAqkgWQi2QBAFUk\nCyAXyQIAqkgWQC6SBQBU0SwAJOEwBMjFYQgAVJEsgFxcKQsAqkgWQC4ufgMAVSQLIBdnQwCgimQB\n5CJZAEAVzQJAEg5DgFwchgBAFckCyEWyAIAqkgWQi2QBAFUkCyAXyQIAqkgWQK7CLn7TSLM46P5o\nYjXTc0rbBWR4bdsFZLin7QJGdInbrqDTSBZALi5+AwBVNAsASTgMAXJx6hQAqkgWQC6SBQBUkSyA\nXCQLAKgiWQC5CpvuTbIAkIRkAeRiujcAVNEsACThMATIxalTAKgamixsv1vS+ZL2S7pV0gUR8R+T\nLgzoPJLFk2yvlfRWSSdHxAslzUn6hcmXBaBrhiWL76o39WS17X2SVkvaM/GqgFnApKwnRcT9kt4n\n6d8k3S3pgYj4p2kUBqBbhh2GPEfSb0haK+lYSYfbfuMU6gK6b98UHx0w7DDkFElfiojvSJLtT0g6\nU9JHn7rYhiXP5/sPoON2L0h7FtquYmYMaxa3Sfp920+X9KiksyV9tbrYhqbrAibv+Pne43E3XTTa\n5zkb8qSI2CbpSklbJN3S//EHJl0UgHy2f872dtv7bJ88YLkjbV9je6ftHbbPGLTeofMsIuJPJP1J\nRs3Aga27yeJWSa+TdOmQ5f5C0qcj4vW2V0k6bNDCTPcGDjARcZsk2SvfYc32MyX9RES8uf+ZRUkP\nDlov072BMv2IpPtsX2H767Yvs7160AdIFkCuSU7KundBum9hxbdtb5J0TM1bF0bEpxK2sErSyZLW\nR8RNti+W9C5J7xn0AQBd80Pzvcfjdj71TE1EnDPmFnZL2h0RN/VfX6Nes1gRzQLI1ZHJUkPUDlxE\nxD22d9l+bkTcrt60iO2DVsSYBXCAsf0627sknSHpetsb+z8/1vb1SxZ9u6SP2t4m6UWS/mjQekkW\nQK6OnjqNiGslXVvz87sl/cyS19sknZq6XpIFgCQkCyBXR5PFpJAsACQhWQC5uPgNAFTRLAAk4TAE\nyDUbk7IaQ7IAkIRkAeTi1CkAVJEsgFyFJYuGmsXmZlYzLVtmsEduqblOcuf9UNsFoEEzuNcAHcGk\nLACoIlkAuZhnAQBVJAsgV2FnQ0gWAJLQLAAk4TAEyMVhCABUkSyAXEzKAoAqkgWQi0lZAFBFsgBy\ncTYEAKpIFkAukgUAVJEsgFzMswCAKpoFgCQchgC5mJQFAFUkCyAXp04BoIpkAeQiWQBAFckCyMWk\nLACoIlkAuZhnAQBVNAsASTgMAXJF2wVMF8kCQBKaBYAkNAsASWgWAJLQLAAkoVkASEKzAJCEeRZA\ntrK+SdbxZPG1tgsY0Za2C8hwe9sFZPhG2wUUqePN4uttFzCiWWtu0mw2i+1tF9C3OMVH+zreLAB0\nBc0CQBJHjPdtGNuFfZ0GB7KIcMpyvf/uH5x0OUs8M7m2SRn7bEjbvwCA6eDUKZCtGwOP08KYBYAk\nJAsgG5OyAKCCZAFkI1kAQAXJAsjWzbMhtv9U0islfV/SNyVdEBG1k0Jsz6n3pabdEfGqQeslWQAH\nnn+U9IKIOEm9L/+8e8Cy75C0QwnXKqdZAAeYiNgUEfv7LzdLOr5uOdvHSzpX0gclDZ1cyWEIkG0m\nBjjfIuljK7z355J+W9IRKSuiWQAzyPYmScfUvHVhRHyqv8zvSvp+RPxNzedfKeneiLjZ9nzKNmkW\nQLZJDnB+tf+oFxHnDPq07V9W7xDjp1ZY5ExJr7Z9rqRDJR1h+8qIeNOK6xz3W6dAiXrfOt0xxS0+\nf5RvxK6T9D5JL4uIbycs/zJJvzXsbAjJAsjW2TGL/ynpEEmbbEvSlyPi12wfK+myiPiZms8MTQ0k\nCyBDL1lsm+IWT2r9chAkCyBbNydlTQrzLAAkIVkA2To7ZjERJAsASUgWQDbGLACggmYBIAmHIUA2\nBjgBoIJkAWRjgBMAKkgWQDbGLACggmQBZGPMAgAqSBZANsYsAKCCZgEgCYchQDYOQwCggmQBZOPU\nKQBUkCyAbIxZAEAFyQLIxpgFAFSQLIBsjFkAQAXJAsjGmAUAVNAsACThMATIxgAnAFSQLIBsDHAC\nQAXJAsjGmAUAVJAsgGyMWQBAhSOi7RqAmWN76jtORHja21yKZgEgCYchAJLQLAAkoVkASEKzAJCE\nZgEgyf8HmzwGuPTq2GoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa0741e8910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = brain.layers[0].Ws[0].eval()\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.matshow(x)\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 11.01352692,  11.28201485,  12.03692055,  12.26954937], dtype=float32)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "brain.input_layer.b.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-2.0"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.collect_reward(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = tf.Variable(tf.zeros((5,5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.ops.Tensor at 0x7f955033c390>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.clip_by_norm(x, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
